import re
import base64
import requests
import time
from bs4 import BeautifulSoup
import pymongo

myclient = pymongo.MongoClient('mongodb://127.0.0.1:27017')
mydb = myclient["darenji_anchors"]
mycol = mydb["anchors"]
pattern = re.compile(r"\d+\.?\d*")

def main(begin, end):

    id = begin

    for item in range(50000):

        id = id + 1

        encodestr = base64.b64encode(str(id).encode('utf-8'))
        url = "http://www.darenji.com/detail/%s.html" % str(encodestr, 'utf-8')
        html = requests.get(url)
        html_text = html.text
        bs = BeautifulSoup(html_text, "html.parser")

        # 主播名称
        anchorName = bs.find("label", {"class": "fs22"})
        anchorName = anchorName.contents[0].string

        if anchorName is not None:
            # 主播id
            anchorId = bs.select_one("div.wxcode-down img")
            anchorId = anchorId.attrs['src']
            anchorId_begin_index = anchorId.find('=')
            anchorId = anchorId[anchorId_begin_index + 1:]
            anchorId = int(anchorId)

            # 主播房间号
            roomNum = bs.select("div.info-li > div:nth-of-type(1) > p:nth-of-type(1)")
            roomNum = roomNum[0].contents[4]
            roomNum = int(roomNum)

            # 所属地区
            location = bs.select("div.info-li > div:nth-of-type(1) > p:nth-of-type(2)")
            location = location[0].contents[4]

            # 粉丝数
            fansNumber = bs.select("div.info-li > div:nth-of-type(2) > p:nth-of-type(1)")
            fansNumber = fansNumber[0].contents[4]
            if fansNumber.find('万') != -1:
                res = re.findall(r'\d+', fansNumber)
                if len(res) > 0:
                    fansNumber = int(res[0]) * 10000
            else:
                fansNumber = int(fansNumber)

            # 认证信息
            identification = bs.select("div.info-li > div:nth-of-type(2) > p:nth-of-type(2)")
            identification = identification[0].contents[4].strip()

            # 标签
            tags = []
            tags_orgin = bs.select(".tagLi a")
            if len(tags_orgin) > 0:
                for item in tags_orgin:
                    tags.append(item.string)

            # 主播简介
            desc = bs.select("div.info-li > p:nth-of-type(1)")
            try:
                desc = desc[0].contents[4]
            except:
                desc = ''

            # 30天直播次数
            month_lives = bs.select_one("#wx > div > div span")
            try:
                res = re.findall(r'\d+', month_lives.contents[0])
                if len(res) > 0:
                    month_lives = int(res[0])
            except:
                month_lives = 0

            # 最近一场直播
            try:
                last_live = bs.select_one("#rank_data a.mg0")
                last_live = last_live.attrs['href']
                last_live_index = last_live.find('=')
                last_live = last_live[last_live_index + 1:]
            except:
                last_live = ''


            insert_obj = {
                "drjId": id,
                "anchorName": anchorName,
                "anchorId": anchorId,
                "roomNum": roomNum,
                "location": location,
                "fansNumber": fansNumber,
                "identification": identification,
                "tags": tags,
                "desc": desc,
                "month_lives": month_lives,
                "last_live": last_live
            }
            print(insert_obj)

            mycol.insert_one(insert_obj)

        else:
            print("{} - {} no data".format(id, url))

        if id >= end:
            break
        time.sleep(1)

    print('end' + str(id))


if __name__ == "__main__":
    main(74763, 80000)
